package net.mohas.docsearch.controller;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.List;

import net.mohas.docsearch.Constants;
import net.mohas.docsearch.model.CustomEnglishAnalyzer;
import net.mohas.docsearch.util.FileUtil;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;

public class IndexingProcess extends DatabaseOperation {

	private static final Logger log = Logger.getLogger(IndexingProcess.class);
	private Analyzer analyzer;

	public IndexingProcess() {
		super();
		analyzer = new CustomEnglishAnalyzer();
	}
	
	public void doIndex(String location) {
		storeFiles(location);
		processFiles();
		calculateTf();
		calculateIdf();
		calculateWeight();
		generateTermDocMatrix();
	}
	
	private void storeFiles(String path) {
		log.debug("Store files ...");
		List<File> listFiles = FileUtil.listAllFiles(path, getDefaultFileFilter());
		for (File file : listFiles) {
			log.debug("File: " + file.getAbsolutePath());
			try {
				String query = "INSERT INTO berkas (namaberkas, lokasiberkas, status) VALUES (?, ?, 0)";
				PreparedStatement ps = connection.prepareStatement(query);
				ps.setString(1, file.getName());
				ps.setString(2, file.getAbsolutePath());
				ps.executeUpdate();
				
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}

	private FileFilter getDefaultFileFilter() {

		FileFilter filter = new FileFilter() {
			public boolean accept(File pathname) {
				if (pathname.isFile()) {
					if (pathname.getName().endsWith(Constants.DOCUMENT_DOC_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_DOCX_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_HTM_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_HTML_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_ODP_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_ODS_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_ODT_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_PDF_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_PPT_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_PPTX_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_RTF_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_TXT_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_XLS_EXT)
							|| pathname.getName().endsWith(Constants.DOCUMENT_XLSX_EXT)) {
					
						return true;
					}
					
				} else if (pathname.isDirectory()) {
					// Directory always accepted except downloaded web files directory (ends with _files)
					if (!pathname.getName().toLowerCase().endsWith("_files")) {						
						return true;
					}
				}
				return false;
			}
		};
		
		return filter;
	}

	private void processFiles() {
		log.debug("Process files ...");
		try {
			connection.setAutoCommit(false);
			PreparedStatement stmtSelectBerkas = connection.prepareStatement("SELECT lokasiberkas, namaberkas, idberkas FROM berkas WHERE status = 0");
			ResultSet resultSelectBerkas = stmtSelectBerkas.executeQuery();
			while (resultSelectBerkas.next()) {
				int idberkas = resultSelectBerkas.getInt("idberkas");
				// String namaberkas = resultSelectBerkas.getString("namaberkas");
	            String lokasiberkas = resultSelectBerkas.getString("lokasiberkas");
	            
	            log.debug("Processing: " + lokasiberkas);
	            
	            TextOperation textOperation = new TextOperation(analyzer);
	            String text = textOperation.getDocumentText(lokasiberkas);
	            List<String> listToken = textOperation.tokenize(analyzer, text);
	            for (int i = 0; i < listToken.size(); i++) {
	            	String kata = listToken.get(i);
	            	
	            	kata = kata.replace("\'", "");

	                // cek apakah kata sudah ada dalam database.
	                // kalau sudah, lakukan UPDATE
	                // kalau belum, lakukan INSERT
	            	PreparedStatement stmtSelectKata = connection.prepareStatement("SELECT kata, jumlah FROM kata WHERE kata= ? AND idberkas = ?");
	            	stmtSelectKata.setString(1, kata);
	            	stmtSelectKata.setInt(2, idberkas);
	            	ResultSet resultSelectKata = stmtSelectKata.executeQuery();
	            	
	            	if (resultSelectKata.next()) {
	            		int jumlah = resultSelectKata.getInt("jumlah");
	            		PreparedStatement stmtUpdateKata = connection.prepareStatement("UPDATE kata SET jumlah = ? WHERE kata = ? AND idberkas = ?");
	            		stmtUpdateKata.setInt(1, (jumlah + 1));
	            		stmtUpdateKata.setString(2, kata);
	            		stmtUpdateKata.setInt(3, idberkas);
	            		stmtUpdateKata.executeUpdate();
	            		
	            	} else {
	            		PreparedStatement stmtInsertKata = connection.prepareStatement("INSERT INTO kata VALUES (?, ?, ?, ?, ?)");
	            		stmtInsertKata.setString(1, kata);
	            		stmtInsertKata.setInt(2, idberkas);
	            		stmtInsertKata.setInt(3, 1);
	            		stmtInsertKata.setDouble(4, 0);
	            		stmtInsertKata.setDouble(5, 0);
	            		stmtInsertKata.executeUpdate();

	            	}
	            	
	            }
	            
	            PreparedStatement stmtUpdateBerkas = connection.prepareStatement("UPDATE berkas SET status = ? WHERE idberkas = ?");
	            stmtUpdateBerkas.setInt(1, 1);
	            stmtUpdateBerkas.setInt(2, idberkas);
	            stmtUpdateBerkas.executeUpdate();
			}
			connection.commit();
			connection.setAutoCommit(true);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
	}

	private void calculateTf() {
		log.debug("Calculate term frequency ...");
		try {			
			PreparedStatement stmtSelectBerkas = connection.prepareStatement("SELECT idberkas FROM berkas");
			ResultSet resultSelectBerkas = stmtSelectBerkas.executeQuery();
			while (resultSelectBerkas.next()) {
				int idberkas = resultSelectBerkas.getInt("idberkas");
				PreparedStatement stmtSelectJumlahKata = connection.prepareStatement("SELECT SUM(jumlah) AS total FROM kata WHERE idberkas = ?");
				stmtSelectJumlahKata.setInt(1, idberkas);
				ResultSet resultSelectJumlahKata = stmtSelectJumlahKata.executeQuery();
				
				while (resultSelectJumlahKata.next()) {
					int total = resultSelectJumlahKata.getInt("total");
					
					PreparedStatement stmtSelectKata = connection.prepareStatement("SELECT kata, jumlah, idberkas FROM kata");
					ResultSet resultSelectKata = stmtSelectKata.executeQuery();
					
					while (resultSelectKata.next()) {
						String kata = resultSelectKata.getString("kata");
						int jumlah = resultSelectKata.getInt("jumlah");
						int currentIdberkas = resultSelectKata.getInt("idberkas");
						double tf = (double) jumlah / total;

						if (currentIdberkas == idberkas) {
							PreparedStatement stmtUpdateKata = connection.prepareStatement("UPDATE kata SET tf = ? WHERE kata = ? AND idberkas = ? ");
							stmtUpdateKata.setDouble(1, tf);
							stmtUpdateKata.setString(2, kata);
							stmtUpdateKata.setInt(3, currentIdberkas);
							stmtUpdateKata.executeUpdate();
						}
					}
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	private void calculateIdf() {
		log.debug("Calculate inverted document frequency ...");
		
		try {			
			PreparedStatement stmtSelectCountBerkas = connection.prepareStatement("SELECT COUNT(idberkas) AS count_berkas FROM berkas");
			ResultSet resultSelectCountBerkas = stmtSelectCountBerkas.executeQuery();
			while (resultSelectCountBerkas.next()) {
				int countBerkas = resultSelectCountBerkas.getInt("count_berkas");
				
				PreparedStatement stmtSelectKata = connection.prepareStatement("SELECT kata, COUNT(kata) AS count_kata FROM kata GROUP BY kata");
				ResultSet resultSelectKata = stmtSelectKata.executeQuery();
				while (resultSelectKata.next()) {
					String kata = resultSelectKata.getString("kata");
					int countKata = resultSelectKata.getInt("count_kata");
					double x = (double) countBerkas / countKata;
					double idf = (double) Math.log10(x);
					
					PreparedStatement stmtInsertIdf = connection.prepareStatement("INSERT INTO df VALUES(?, ?, ?)");
					stmtInsertIdf.setString(1, kata);;
					stmtInsertIdf.setInt(2, countKata);
					stmtInsertIdf.setDouble(3, idf);
					stmtInsertIdf.executeUpdate();
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		
	}
	
	private void calculateWeight() {
		log.debug("Calculate weight ...");
		
		try {
			PreparedStatement stmtSelectKata = connection.prepareStatement("SELECT kata, idberkas, tf FROM kata");
			ResultSet resultSelectKata = stmtSelectKata.executeQuery();
			while (resultSelectKata.next()) {
				String kata = resultSelectKata.getString("kata");
				int idberkas = resultSelectKata.getInt("idberkas");
				double tf = resultSelectKata.getDouble("tf");
				
				PreparedStatement stmtSelectDf = connection.prepareStatement("SELECT idf FROM df WHERE kata = ?");
				stmtSelectDf.setString(1, kata);
				ResultSet resultSelectDf = stmtSelectDf.executeQuery();
				while (resultSelectDf.next()) {
					double idf = resultSelectDf.getDouble("idf");
					double bobot = tf * idf;
					log.debug("tf: " + tf + ", idf: " + idf + ", bobot: " + bobot);
					
					PreparedStatement stmtUpdateKata = connection.prepareStatement("UPDATE kata SET bobot = ? WHERE kata = ? AND idberkas = ?");
					stmtUpdateKata.setDouble(1, bobot);
					stmtUpdateKata.setString(2, kata);
					stmtUpdateKata.setInt(3, idberkas);
					stmtUpdateKata.executeUpdate();
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	private void generateTermDocMatrix() {
		log.debug("Generate term document matrix ...");
		
		Writer writer = null;
		try {
			File file = new File(Constants.RESOURCES_DIR
					+ Constants.MATRIX_DIR
					+ Constants.TERM_DOC_MATRIX_FILENAME);
				
			writer = new BufferedWriter(new FileWriter(file));
			
			PreparedStatement stmtCountBerkas = connection.prepareStatement("SELECT COUNT(idberkas) AS count FROM berkas");
			ResultSet resultCountBerkas = stmtCountBerkas.executeQuery();
			while (resultCountBerkas.next()) {
//				int count = resultCountBerkas.getInt("count");
//				
//				PreparedStatement stmtOrderDf = connection.prepareStatement("ALTER TABLE df ORDER BY kata");
//				stmtOrderDf.executeUpdate();
				
				PreparedStatement stmtSelectDf = connection.prepareStatement("SELECT kata FROM df ORDER BY kata");
				ResultSet resultSelectDf = stmtSelectDf.executeQuery();
				while (resultSelectDf.next()) {
					String kata = resultSelectDf.getString("kata");
					
					PreparedStatement stmtSelectBerkas = connection.prepareStatement("SELECT idberkas FROM berkas");
					ResultSet resultSelectBerkas = stmtSelectBerkas.executeQuery();
					while (resultSelectBerkas.next()) {
						int idberkas = resultSelectBerkas.getInt("idberkas");
						
						PreparedStatement stmtSelectBobotKata = connection.prepareStatement("SELECT bobot FROM kata WHERE kata = ? AND idberkas = ?");
						stmtSelectBobotKata.setString(1, kata);
						stmtSelectBobotKata.setInt(2, idberkas);
						ResultSet resultSelectBobotKata = stmtSelectBobotKata.executeQuery();
						if (resultSelectBobotKata.next()) {
							double bobot = resultSelectBobotKata.getDouble("bobot");
							writer.append(String.format("%1.16f", bobot));
							writer.append(" ");
						} else {
							writer.append(String.format("%1.16f", 0.0));
							writer.append(" ");
						}
					}
					writer.append("\n");
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
			
		} finally {
			
			if (writer != null) {
				try {
					writer.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			
		}
	}
	
	public Analyzer getAnalyzer() {
		return analyzer;
	}

	public void setAnalyzer(Analyzer analyzer) {
		this.analyzer = analyzer;
	}

	public static void main(String[] args) {
		IndexingProcess indexing = new IndexingProcess();
		indexing.doIndex("D:\\My Files\\ECLIPSEDOCS\\doc-search-app\\documents\\Inggris\\Percobaan 2");
	}
	
}
